package com.jiuzhou.crawlers.taobao.crawler.taobao;


import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import javax.management.JMException;

import org.apache.hadoop.fs.Path;

import com.jiuzhou.crawlers.taobao.pipeline.taobao.TaoBaoPipeline;
import com.jiuzhou.crawlers.taobao.pipeline.taobao.TaobaoItemDetailPipeline;
import com.jiuzhou.crawlers.taobao.processor.taobao.ItemDetailTaoBaoProcessor;
import com.jiuzhou.crawlers.taobao.processor.taobao.constant.CommonConstant;
import com.jiuzhou.crawlers.taobao.util.DataGetUtil;
import com.jiuzhou.crawlers.taobao.util.HdfsUtil;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
	/**
	 * 项目名称：大数据爬虫
	 * @author yanganzhen
	 * version-code：1.0
     * 类说明：商品详情爬虫启动类
     * 创建时间：2016年12月13日
	 */
public class ItemDetailStart {

	
	public static void main(String[] args) throws IOException, JMException {
           CommonConstant.CURRENTTYPE="";
		  Spider oschinaSpider = Spider.create(new ItemDetailTaoBaoProcessor())
		  .addPipeline(new TaobaoItemDetailPipeline("hdfs://master:9000/user/yanganzhenGoodsDetail/allsorts")).addUrl("https://www.taobao.com").thread(20); 
		  //JVM监控
		  SpiderMonitor.instance().register(oschinaSpider);
		  oschinaSpider.start();
	}

}
